{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "couldn't import doomish\n",
      "Couldn't import doom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:174: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:187: UserWarning: \u001b[33mWARN: Future gym versions will require that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:195: UserWarning: \u001b[33mWARN: The result returned by `env.reset()` was not a tuple of the form `(obs, info)`, where `obs` is a observation and `info` is a dictionary containing additional information. Actual type: `<class 'numpy.ndarray'>`\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([24.,  0.,  7., 17., 49., 31., 40.])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "import gym_pygame\n",
    "\n",
    "#!pip install git+https://github.com/ntasfi/PyGame-Learning-Environment.git\n",
    "#!pip install git+https://github.com/qlan3/gym-games.git\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('Pixelcopter-PLE-v0')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box(-inf, inf, (7,), float32)\n",
      "env.action_space= Discrete(2)\n",
      "state= [24.  0.  7. 17. 46. 23. 32.]\n",
      "action= 1\n",
      "next_state= [24.057024  0.057024  7.057024 16.942976 45.36     23.       32.      ]\n",
      "reward= 0.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:219: DeprecationWarning: \u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to rewrite the environment with new step API. \u001b[0m\n",
      "  logger.deprecation(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:141: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method was expecting numpy array dtype to be float32, actual type: float64\u001b[0m\n",
      "  logger.warn(\n",
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:165: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method is not within the observation space.\u001b[0m\n",
      "  logger.warn(f\"{pre} is not within the observation space.\")\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/gym/utils/passive_env_checker.py:272: UserWarning: \u001b[33mWARN: No render modes was declared in the environment (env.metadata['render_modes'] is None or not defined), you may have trouble when calling `.render()`.\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARYAAAEVCAYAAAAl2crhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAATjklEQVR4nO3df2jV1/3H8VfS5F5rk9wktr23wdwu0NLQiRlN1V46GDNBESlq8kcH/SPUsqLeiCaDzcBq/xnc0EJr7WxX1s1uf2wZKWixo50h1itj0ZmroVZt1oHMC8m9Wf+4P5qam5B7vn/027veGn/ceOL9xD4f8AZzPief+/4E8uJ4Pp/cW2KMMQIAi0qL3QCAOw/BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsC6soU68YEDB/Tyyy8rFoupqalJr7/+ulavXn3D78tmsxobG1NlZaVKSkoWqj0ABTLGKJ1Oq66uTqWlN1iTmAXQ19dnXC6X+f3vf2/Onz9vfvrTn5rq6moTj8dv+L3RaNRIoijKoRWNRm/4e1xijP0/QlyzZo1WrVqlX//615K+WoXU19dr586d2rNnz3W/N5lMqrq6WjvOr5O7stx2awDmKZOe0RvfP6pEIiGPx3Pdudb/KzQ9Pa1IJKKenp7cWGlpqVpbWzU0NHR1s5mMMplM7ut0Oi1JcleWy11FsABOczNbFNY3bz///HPNzs7K6/XmjXu9XsVisavmh0IheTyeXNXX19tuCcBtVvS7Qj09PUomk7mKRqPFbgnALbL+X6F7771Xd911l+LxeN54PB6Xz+e7ar7b7Zbb7bbdBoAisr5icblcam5u1uDgYG4sm81qcHBQgUDA9ssBcKAFeY6lu7tbHR0devzxx7V69Wrt27dPk5OTevbZZxfi5QA4zIIEy9NPP63//ve/2rt3r2KxmH7wgx/oww8/vGpDF8CdacGevO3s7FRnZ+dCnR6AgxX9rhCAOw/BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6ggWAdQv2nrcAbr/e6sPFbkESKxYAC4BgAWAdwQLAOoIFgHVs3gJF5pQNV5tYsQCwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFjHA3JAge7EB9psY8UCwDqCBYB1BAsA6woOlhMnTuipp55SXV2dSkpKdPjw4bzjxhjt3btXDzzwgO6++261trbqs88+s9UvgEWg4M3byclJNTU1aevWrWpra7vq+EsvvaT9+/frD3/4gxoaGvTCCy9o/fr1unDhgpYsWWKlaaBQbLjeXgUHy4YNG7Rhw4Y5jxljtG/fPv3yl7/Upk2bJEl//OMf5fV6dfjwYf3kJz+5tW4BLApW91guXbqkWCym1tbW3JjH49GaNWs0NDQ05/dkMhmlUqm8ArC4WQ2WWCwmSfJ6vXnjXq83d+zbQqGQPB5Prurr6222BKAIin5XqKenR8lkMlfRaLTYLQG4RVaDxefzSZLi8XjeeDwezx37NrfbraqqqrwCsLhZDZaGhgb5fD4NDg7mxlKplE6dOqVAIGDzpQA4WMF3hb744gv9+9//zn196dIljYyMqLa2Vn6/X7t379avfvUrPfzww7nbzXV1ddq8ebPNvgE4WMHBMjw8rB//+Me5r7u7uyVJHR0deuedd/Tzn/9ck5OTev7555VIJPTDH/5QH374Ic+wAN8hJcYYU+wmvimVSsnj8ajr8ka5q8qL3Q7uEDwgZ08ymbzhXmjR7woBuPMQLACsI1gAWEewALCOYAFgHe95C6u4+wKJFQuABUCwALCOYAFgnWP3WF71/7Xg79mT2Gy/EQAFY8UCwDqCBYB1BAsA6wgWANY5dvN2Pmw/nMVmMDA/rFgAWEewALCOYAFgHcECwLo7avPWNpubwWwE47uEFQsA6wgWANYRLACsI1gAWOfYDywD4Ex8YBmAoiBYAFhHsACwjmABYB3BAsA6ggWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYF1BwRIKhbRq1SpVVlbq/vvv1+bNmzU6Opo3Z2pqSsFgUMuWLVNFRYXa29sVj8etNg3A2QoKlnA4rGAwqJMnT2pgYEAzMzNat26dJicnc3O6urp05MgR9ff3KxwOa2xsTG1tbdYbB+Bg5hZMTEwYSSYcDhtjjEkkEqa8vNz09/fn5ly8eNFIMkNDQzd1zmQyaSRRFOXQSiaTN/w9vqU9lmQyKUmqra2VJEUiEc3MzKi1tTU3p7GxUX6/X0NDQ3OeI5PJKJVK5RWAxW3ewZLNZrV79249+eSTWrFihSQpFovJ5XKpuro6b67X61UsFpvzPKFQSB6PJ1f19fXzbQmAQ8w7WILBoD755BP19fXdUgM9PT1KJpO5ikajt3Q+AMU3r88V6uzs1Pvvv68TJ05o+fLluXGfz6fp6WklEom8VUs8HpfP55vzXG63W263ez5tAHCqQjZrs9msCQaDpq6uzvzrX/+66vjXm7fvvvtubuzTTz81Epu3FHWn1M1s3hYULNu3bzcej8ccP37cjI+P5+rLL7/Mzdm2bZvx+/3m2LFjZnh42AQCARMIBG76NQgWinJ2WQ+Wa73QwYMHc3OuXLliduzYYWpqaszSpUvNli1bzPj4OMFCUXdI3Uyw8PEfAArCx38AKAqCBYB1BAsA6wgWANYRLACsm9eTtyium72RV1JSssCdAHNjxQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdD8gtQjz4BqdjxQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYR7AAsI5gAWAdwQLAOoIFgHUECwDrCBYA1hEsAKwjWABYV1CwvPnmm1q5cqWqqqpUVVWlQCCgDz74IHd8ampKwWBQy5YtU0VFhdrb2xWPx603DcDZCgqW5cuXq7e3V5FIRMPDw1q7dq02bdqk8+fPS5K6urp05MgR9ff3KxwOa2xsTG1tbQvSOAAHM7eopqbGvP322yaRSJjy8nLT39+fO3bx4kUjyQwNDd30+ZLJpJFEUZRDK5lM3vD3eN57LLOzs+rr69Pk5KQCgYAikYhmZmbU2tqam9PY2Ci/36+hoaFrnieTySiVSuUVgMWt4GA5d+6cKioq5Ha7tW3bNh06dEiPPvqoYrGYXC6Xqqur8+Z7vV7FYrFrni8UCsnj8eSqvr6+4IsA4CwFB8sjjzyikZERnTp1Stu3b1dHR4cuXLgw7wZ6enqUTCZzFY1G530uAM5Q8IfCu1wuPfTQQ5Kk5uZmnT59Wq+99pqefvppTU9PK5FI5K1a4vG4fD7fNc/ndrvldrsL7xyAY93ycyzZbFaZTEbNzc0qLy/X4OBg7tjo6KguX76sQCBwqy8DYBEpaMXS09OjDRs2yO/3K51O609/+pOOHz+uv/3tb/J4PHruuefU3d2t2tpaVVVVaefOnQoEAnriiScWqn8ATlTIreWtW7eaBx980LhcLnPfffeZlpYWc/To0dzxK1eumB07dpiamhqzdOlSs2XLFjM+Pl7IS3C7maIcXjdzu7nEGGPkIKlUSh6Pp9htALiGZDKpqqqq684pePMWgHPtSWxesHNnUjN61f/Xm5rLHyECsI5gAWAdwQLAOoIFgHVs3gJFtpAbrsXCigWAdQQLAOsIFgDWESwArCNYAFhHsACwjmABYB3BAsA6HpADCnQnPtBmGysWANYRLACsI1gAWEewALCOzVt8J7DhenuxYgFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACs420TYBVvTwCJFQuABUCwALCOYAFg3S3tsfT29qqnp0e7du3Svn37JElTU1P62c9+pr6+PmUyGa1fv15vvPGGvF5vQefuurxR7qrywvqpPlzQfAALY94rltOnT+utt97SypUr88a7urp05MgR9ff3KxwOa2xsTG1tbbfcKIDFY17B8sUXX+iZZ57Rb3/7W9XU1OTGk8mkfve73+mVV17R2rVr1dzcrIMHD+of//iHTp48Oee5MpmMUqlUXgFY3OYVLMFgUBs3blRra2veeCQS0czMTN54Y2Oj/H6/hoaG5jxXKBSSx+PJVX19/XxaAuAgBQdLX1+fzpw5o1AodNWxWCwml8ul6urqvHGv16tYLDbn+Xp6epRMJnMVjUYLbQmAwxS0eRuNRrVr1y4NDAxoyZIlVhpwu91yu91WzmX74Sw2g4H5KWjFEolENDExoccee0xlZWUqKytTOBzW/v37VVZWJq/Xq+npaSUSibzvi8fj8vl8NvsG4GAFrVhaWlp07ty5vLFnn31WjY2N+sUvfqH6+nqVl5drcHBQ7e3tkqTR0VFdvnxZgUDAXtcAHK2gYKmsrNSKFSvyxu655x4tW7YsN/7cc8+pu7tbtbW1qqqq0s6dOxUIBPTEE0/Y6xqAo1n/I8RXX31VpaWlam9vz3tADsB3R4kxxhS7iW9KpVLyeDzzevLWyb4rG8H8dfOdK5Oa0av+vyqZTKqqquq6c/lbIQDWESwArCNYAFhHsACwjremvE3Y1MR3CSsWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDrHffzH1x8lnUnPFLkTAN/09e/kzXzcu+OCJZ1OS5Le+P7RIncCYC7pdFoej+e6c0rMzcTPbZTNZjU2NqbKykql02nV19crGo3e8NPtnSiVStF/EdG/XcYYpdNp1dXVqbT0+rsojluxlJaWavny5ZKkkpISSVJVVZUjfrDzRf/FRf/23Gil8jU2bwFYR7AAsM7RweJ2u/Xiiy/K7XYXu5V5of/iov/icdzmLYDFz9ErFgCLE8ECwDqCBYB1BAsA6wgWANY5NlgOHDig733ve1qyZInWrFmjf/7zn8Vu6ZpOnDihp556SnV1dSopKdHhw4fzjhtjtHfvXj3wwAO6++671draqs8++6w4zX5LKBTSqlWrVFlZqfvvv1+bN2/W6Oho3pypqSkFg0EtW7ZMFRUVam9vVzweL1LH+d58802tXLky93RqIBDQBx98kDvu5N7n0tvbq5KSEu3evTs3ttiuQXJosPzlL39Rd3e3XnzxRZ05c0ZNTU1av369JiYmit3anCYnJ9XU1KQDBw7Mefyll17S/v379Zvf/EanTp3SPffco/Xr12tqauo2d3q1cDisYDCokydPamBgQDMzM1q3bp0mJydzc7q6unTkyBH19/crHA5rbGxMbW1tRez6f5YvX67e3l5FIhENDw9r7dq12rRpk86fPy/J2b1/2+nTp/XWW29p5cqVeeOL6RpyjAOtXr3aBIPB3Nezs7Omrq7OhEKhInZ1cySZQ4cO5b7OZrPG5/OZl19+OTeWSCSM2+02f/7zn4vQ4fVNTEwYSSYcDhtjvuq1vLzc9Pf35+ZcvHjRSDJDQ0PFavO6ampqzNtvv72oek+n0+bhhx82AwMD5kc/+pHZtWuXMWZx/vyNMcZxK5bp6WlFIhG1trbmxkpLS9Xa2qqhoaEidjY/ly5dUiwWy7sej8ejNWvWOPJ6ksmkJKm2tlaSFIlENDMzk9d/Y2Oj/H6/4/qfnZ1VX1+fJicnFQgEFlXvwWBQGzduzOtVWlw//29y3F83f/7555qdnZXX680b93q9+vTTT4vU1fzFYjFJmvN6vj7mFNlsVrt379aTTz6pFStWSPqqf5fLperq6ry5Tur/3LlzCgQCmpqaUkVFhQ4dOqRHH31UIyMjju9dkvr6+nTmzBmdPn36qmOL4ec/F8cFC4onGAzqk08+0d///vdit1KQRx55RCMjI0omk3r33XfV0dGhcDhc7LZuSjQa1a5duzQwMKAlS5YUux1rHPdfoXvvvVd33XXXVbve8XhcPp+vSF3N39c9O/16Ojs79f777+ujjz7KvR+O9FX/09PTSiQSefOd1L/L5dJDDz2k5uZmhUIhNTU16bXXXlsUvUciEU1MTOixxx5TWVmZysrKFA6HtX//fpWVlcnr9Tr+GubiuGBxuVxqbm7W4OBgbiybzWpwcFCBQKCInc1PQ0ODfD5f3vWkUimdOnXKEddjjFFnZ6cOHTqkY8eOqaGhIe94c3OzysvL8/ofHR3V5cuXHdH/XLLZrDKZzKLovaWlRefOndPIyEiuHn/8cT3zzDO5fzv9GuZU7N3jufT19Rm3223eeecdc+HCBfP888+b6upqE4vFit3anNLptDl79qw5e/askWReeeUVc/bsWfOf//zHGGNMb2+vqa6uNu+99575+OOPzaZNm0xDQ4O5cuVKkTs3Zvv27cbj8Zjjx4+b8fHxXH355Ze5Odu2bTN+v98cO3bMDA8Pm0AgYAKBQBG7/p89e/aYcDhsLl26ZD7++GOzZ88eU1JSYo4ePWqMcXbv1/LNu0LGLM5rcGSwGGPM66+/bvx+v3G5XGb16tXm5MmTxW7pmj766CMj6arq6Ogwxnx1y/mFF14wXq/XuN1u09LSYkZHR4vb9P+bq29J5uDBg7k5V65cMTt27DA1NTVm6dKlZsuWLWZ8fLx4TX/D1q1bzYMPPmhcLpe57777TEtLSy5UjHF279fy7WBZjNfA+7EAsM5xeywAFj+CBYB1BAsA6wgWANYRLACsI1gAWEewALCOYAFgHcECwDqCBYB1BAsA6/4PjywHnKQwUY8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render(mode='rgb_array'))\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x7f95fc339f40>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = PPO(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=1024,  #运行N步后执行更新,buffer_size=n_steps*环境数量\n",
    "    batch_size=64,  #采样数据量\n",
    "    n_epochs=16,  #每次采样后训练的次数\n",
    "    gamma=0.99,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-4.95, 0.21794494717703367)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/2.PPO.Pixelcopter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(27.1, 19.506152875439074)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = PPO.load('save/2.PPO.Pixelcopter')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARYAAAEVCAYAAAAl2crhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAT80lEQVR4nO3df2yU9QHH8U9rewfS3rVFuKOhdU00do5QY+XHxWU/aENDiAHpEpf4R6dkRrkSoEs2mkz8Z8k1mgjiAM3cMPuDdemSYnDR0RQ9sqx0cNCIqI1LyLik3KF/9O7s6LVpv/vDefOkQK98yz2H71fyTezzPPf08zTyybff57lrkTHGCAAsKs53AAB3HooFgHUUCwDrKBYA1lEsAKyjWABYR7EAsI5iAWAdxQLAOooFgHUl83XiAwcO6KWXXlIsFlNDQ4NeffVVrV69+qavm56e1sjIiMrLy1VUVDRf8QDkyBijVCql6upqFRffZE5i5kF3d7dxuVzmD3/4g7lw4YL5+c9/bioqKkw8Hr/pa6PRqJHEYDAcOqLR6E3/HRcZY/9NiGvWrNGqVav029/+VtKXs5Camhpt375du3fvvuFrE4mEKioqtO3CernLS21HAzBH6dSkDn7vuEZHR+X1em94rPVfhSYmJhSJRNTZ2ZnZVlxcrObmZg0MDFwbNp1WOp3OfJ1KpSRJ7vJSuT0UC+A0s1misL54+/nnn2tqako+ny9ru8/nUywWu+b4UCgkr9ebGTU1NbYjAbjN8n5XqLOzU4lEIjOi0Wi+IwG4RdZ/Fbrnnnt01113KR6PZ22Px+Py+/3XHO92u+V2u23HAJBH1mcsLpdLjY2N6u/vz2ybnp5Wf3+/AoGA7W8HwIHm5TmWjo4OtbW16ZFHHtHq1au1b98+jY2N6amnnpqPbwfAYealWJ544gl99tln2rNnj2KxmB566CG9++671yzoArgzzduTt+3t7Wpvb5+v0wNwsLzfFQJw56FYAFhHsQCwjmIBYB3FAsA6igWAdRQLAOsoFgDWUSwArKNYAFhHsQCwjmIBYB3FAsA6igWAdRQLAOsoFgDWUSwArKNYAFhHsQCwbt4+8xZA4eiqOGr1fMxYAFhHsQCwjmIBYB3FAsC6ImOMyXeIr0smk/J6vbM6dvfo5vkNAziY7QXX2UokEvJ4PDc8hhkLAOsoFgDWUSwArKNYAFhHsQCwjmIBYB3FAsA6igWAdby7GbiN8vVQ2+3GjAWAdRQLAOsoFgDW5VwsJ0+e1GOPPabq6moVFRXp6NGjWfuNMdqzZ4+WLVumhQsXqrm5WZ9++qmtvAAKQM7FMjY2poaGBh04cGDG/S+++KL279+v1157TYODg1q0aJFaWlo0Pj5+y2EBFIac7wpt2LBBGzZsmHGfMUb79u3Tr3/9a23atEmS9Mc//lE+n09Hjx7VT3/601tLC6AgWF1juXjxomKxmJqbmzPbvF6v1qxZo4GBgRlfk06nlUwmswaAwma1WGKxmCTJ5/Nlbff5fJl93xQKheT1ejOjpqbGZiQAeZD3u0KdnZ1KJBKZEY1G8x0JwC2y+uSt3++XJMXjcS1btiyzPR6P66GHHprxNW63W263e07fz+ZTjHzMJWCP1RlLXV2d/H6/+vv7M9uSyaQGBwcVCARsfisADpbzjOWLL77Qv/71r8zXFy9e1NDQkKqqqlRbW6udO3fqN7/5je6//37V1dXp+eefV3V1tTZv3mwzNwAHy7lYzpw5ox//+MeZrzs6OiRJbW1tevPNN/XLX/5SY2NjeuaZZzQ6Oqrvf//7evfdd7VgwQJ7qQE4WkH/+Q+bWGPB7XAnvLuZP/8BIC8oFgDWUSwArKNYAFhHsQCwjs+8BW5irndyZrrTONe7j4V2N4kZCwDrKBYA1lEsAKxjjeV/eKc0YA8zFgDWUSwArKNYAFhHsQCwjsXbecBCcH4V2sNkdyJmLACso1gAWEexALCOYgFgHYu3Dmd7IdKpi8EsuN5ZmLEAsI5iAWAdxQLAOooFgHUs3n7LsEiK24EZCwDrKBYA1lEsAKyjWABYR7EAsI5iAWAdxQLAOooFgHUUCwDrKBYA1lEsAKyjWABYl1OxhEIhrVq1SuXl5Vq6dKk2b96s4eHhrGPGx8cVDAa1ePFilZWVqbW1VfF43GpoAM6WU7GEw2EFg0GdOnVKfX19mpyc1Pr16zU2NpY5ZteuXTp27Jh6enoUDoc1MjKiLVu2WA8OwLmKjDFmri/+7LPPtHTpUoXDYf3gBz9QIpHQkiVLdOTIEf3kJz+RJH3yySf67ne/q4GBAa1du/am50wmk/J6vXONBDiGzc8XdtLHXSQSCXk8nhsec0trLIlEQpJUVVUlSYpEIpqcnFRzc3PmmPr6etXW1mpgYGDGc6TTaSWTyawBoLDNuVimp6e1c+dOPfroo1qxYoUkKRaLyeVyqaKiIutYn8+nWCw243lCoZC8Xm9m1NTUzDUSAIeYc7EEg0F9+OGH6u7uvqUAnZ2dSiQSmRGNRm/pfADyb04fTdne3q63335bJ0+e1PLlyzPb/X6/JiYmNDo6mjVricfj8vv9M57L7XbL7XbPJQYAh8ppxmKMUXt7u3p7e3XixAnV1dVl7W9sbFRpaan6+/sz24aHh3Xp0iUFAgE7iQE4Xk4zlmAwqCNHjuitt95SeXl5Zt3E6/Vq4cKF8nq92rp1qzo6OlRVVSWPx6Pt27crEAjM6o4QgDtDTsVy6NAhSdKPfvSjrO2HDx/Wz372M0nS3r17VVxcrNbWVqXTabW0tOjgwYNWwgIoDLf0HMt84DkW3Cm+zc+xUCwAcjLvD8gBwEwoFgDWUSwArKNYAFhHsQCwjmIBYB3FAsA6igWAdRQLAOsoFgDWUSwArKNYAFhHsQCwjmIBYB3FAsA6igWAdRQLAOsoFgDWUSwArKNYAFhHsQCwjmIBYB3FAsA6igWAdRQLAOsoFgDWUSwArKNYAFhHsQCwjmIBYB3FAsA6igWAdRQLAOtK8h0AuTPGzOq4oqKieU4CzIwZCwDrKBYA1uVULIcOHdLKlSvl8Xjk8XgUCAT0zjvvZPaPj48rGAxq8eLFKisrU2trq+LxuPXQAJwtp2JZvny5urq6FIlEdObMGa1bt06bNm3ShQsXJEm7du3SsWPH1NPTo3A4rJGREW3ZsmVeggNwMHOLKisrzRtvvGFGR0dNaWmp6enpyez7+OOPjSQzMDAw6/MlEgkjiXGDMVv5zsm4M0cikbjp/3tzXmOZmppSd3e3xsbGFAgEFIlENDk5qebm5swx9fX1qq2t1cDAwHXPk06nlUwmswaAwpZzsZw/f15lZWVyu9169tln1dvbqwcffFCxWEwul0sVFRVZx/t8PsViseueLxQKyev1ZkZNTU3OFwHAWXIulgceeEBDQ0MaHBzUc889p7a2Nn300UdzDtDZ2alEIpEZ0Wh0zucC4Aw5PyDncrl03333SZIaGxt1+vRpvfLKK3riiSc0MTGh0dHRrFlLPB6X3++/7vncbrfcbnfuyb/FePANTnfLz7FMT08rnU6rsbFRpaWl6u/vz+wbHh7WpUuXFAgEbvXbACggOc1YOjs7tWHDBtXW1iqVSunIkSN6//339be//U1er1dbt25VR0eHqqqq5PF4tH37dgUCAa1du3a+8gNwolnfuzTGPP300+bee+81LpfLLFmyxDQ1NZnjx49n9l+9etVs27bNVFZWmrvvvts8/vjj5vLly7l8C243MxgOH7O53VxkzCzf0XabJJNJeb3efMcAcB2JREIej+eGx/DuZgDaPbr5psekk5PaW/vXWZ2PNyECsI5iAWAdxQLAOooFgHWOXbzddWmj3J7SGx7TVXH09oQBHGg2C675wowFgHUUCwDrKBYA1lEsAKyjWABYR7EAsI5iAWAdxQLAOsc+IAfciZz8UJtNzFgAWEexALCOYgFgHcUCwDqKBYB1FAsA6ygWANZRLACso1gAWFfQT97afIqRj7kE7GHGAsA6igWAdRQLAOsoFgDWUSwArKNYAFhHsQCwjmIBYB3FAsC6gn7yFrgdvi2fU2sTMxYA1lEsAKyjWABYd0trLF1dXers7NSOHTu0b98+SdL4+Lh+8YtfqLu7W+l0Wi0tLTp48KB8Pp+NvPOGd0oD9sx5xnL69Gm9/vrrWrlyZdb2Xbt26dixY+rp6VE4HNbIyIi2bNlyy0EBFI45FcsXX3yhJ598Ur/73e9UWVmZ2Z5IJPT73/9eL7/8statW6fGxkYdPnxY//jHP3Tq1KkZz5VOp5VMJrMGgMI2p2IJBoPauHGjmpubs7ZHIhFNTk5mba+vr1dtba0GBgZmPFcoFJLX682MmpqauUQC4CA5F0t3d7fOnj2rUCh0zb5YLCaXy6WKioqs7T6fT7FYbMbzdXZ2KpFIZEY0Gs01EgCHyWnxNhqNaseOHerr69OCBQusBHC73XK73VbO5RQsBOcXD7TlX04zlkgkoitXrujhhx9WSUmJSkpKFA6HtX//fpWUlMjn82liYkKjo6NZr4vH4/L7/TZzA3CwnGYsTU1NOn/+fNa2p556SvX19frVr36lmpoalZaWqr+/X62trZKk4eFhXbp0SYFAwF5qAI6WU7GUl5drxYoVWdsWLVqkxYsXZ7Zv3bpVHR0dqqqqksfj0fbt2xUIBLR27Vp7qQE4mvU3Ie7du1fFxcVqbW3NekAOwLdHkTHG5DvE1yWTSXm9Xu26tFFuT2m+49xxnLoYzIKr86WTk9pb+1clEgl5PJ4bHst7hQBYR7EAsI5iAWAdxQLAOj6a8luGRVLcDsxYAFhHsQCwjmIBYB3FAsA6igWAdRQLAOsoFgDWUSwArKNYAFhHsQCwjmIBYB3FAsA6igWAdRQLAOsoFgDWUSwArKNYAFhHsQCwjmIBYB3FAsA6igWAdRQLAOsc9+c/vvpT0unUZJ6TAPi6r/5NzubPvTuuWFKplCTp4PeO5zkJgJmkUil5vd4bHlNkZlM/t9H09LRGRkZUXl6uVCqlmpoaRaPRm/51eydKJpPkzyPy22WMUSqVUnV1tYqLb7yK4rgZS3FxsZYvXy5JKioqkiR5PB5H/GDnivz5RX57bjZT+QqLtwCso1gAWOfoYnG73XrhhRfkdrvzHWVOyJ9f5M8fxy3eAih8jp6xAChMFAsA6ygWANZRLACso1gAWOfYYjlw4IC+853vaMGCBVqzZo3++c9/5jvSdZ08eVKPPfaYqqurVVRUpKNHj2btN8Zoz549WrZsmRYuXKjm5mZ9+umn+Qn7DaFQSKtWrVJ5ebmWLl2qzZs3a3h4OOuY8fFxBYNBLV68WGVlZWptbVU8Hs9T4myHDh3SypUrM0+nBgIBvfPOO5n9Ts4+k66uLhUVFWnnzp2ZbYV2DZJDi+XPf/6zOjo69MILL+js2bNqaGhQS0uLrly5ku9oMxobG1NDQ4MOHDgw4/4XX3xR+/fv12uvvabBwUEtWrRILS0tGh8fv81JrxUOhxUMBnXq1Cn19fVpcnJS69ev19jYWOaYXbt26dixY+rp6VE4HNbIyIi2bNmSx9T/t3z5cnV1dSkSiejMmTNat26dNm3apAsXLkhydvZvOn36tF5//XWtXLkya3shXUOGcaDVq1ebYDCY+XpqaspUV1ebUCiUx1SzI8n09vZmvp6enjZ+v9+89NJLmW2jo6PG7XabP/3pT3lIeGNXrlwxkkw4HDbGfJm1tLTU9PT0ZI75+OOPjSQzMDCQr5g3VFlZad54442Cyp5Kpcz9999v+vr6zA9/+EOzY8cOY0xh/vyNMcZxM5aJiQlFIhE1NzdnthUXF6u5uVkDAwN5TDY3Fy9eVCwWy7oer9erNWvWOPJ6EomEJKmqqkqSFIlENDk5mZW/vr5etbW1jss/NTWl7u5ujY2NKRAIFFT2YDCojRs3ZmWVCuvn/3WOe3fz559/rqmpKfl8vqztPp9Pn3zySZ5SzV0sFpOkGa/nq31OMT09rZ07d+rRRx/VihUrJH2Z3+VyqaKiIutYJ+U/f/68AoGAxsfHVVZWpt7eXj344IMaGhpyfHZJ6u7u1tmzZ3X69Olr9hXCz38mjisW5E8wGNSHH36ov//97/mOkpMHHnhAQ0NDSiQS+stf/qK2tjaFw+F8x5qVaDSqHTt2qK+vTwsWLMh3HGsc96vQPffco7vuuuuaVe94PC6/35+nVHP3VWanX097e7vefvttvffee5nPw5G+zD8xMaHR0dGs452U3+Vy6b777lNjY6NCoZAaGhr0yiuvFET2SCSiK1eu6OGHH1ZJSYlKSkoUDoe1f/9+lZSUyOfzOf4aZuK4YnG5XGpsbFR/f39m2/T0tPr7+xUIBPKYbG7q6urk9/uzrieZTGpwcNAR12OMUXt7u3p7e3XixAnV1dVl7W9sbFRpaWlW/uHhYV26dMkR+WcyPT2tdDpdENmbmpp0/vx5DQ0NZcYjjzyiJ598MvPfTr+GGeV79Xgm3d3dxu12mzfffNN89NFH5plnnjEVFRUmFovlO9qMUqmUOXfunDl37pyRZF5++WVz7tw58+9//9sYY0xXV5epqKgwb731lvnggw/Mpk2bTF1dnbl69Wqekxvz3HPPGa/Xa95//31z+fLlzPjPf/6TOebZZ581tbW15sSJE+bMmTMmEAiYQCCQx9T/t3v3bhMOh83FixfNBx98YHbv3m2KiorM8ePHjTHOzn49X78rZExhXoMji8UYY1599VVTW1trXC6XWb16tTl16lS+I13Xe++9ZyRdM9ra2owxX95yfv75543P5zNut9s0NTWZ4eHh/Ib+n5lySzKHDx/OHHP16lWzbds2U1lZae6++27z+OOPm8uXL+cv9Nc8/fTT5t577zUul8ssWbLENDU1ZUrFGGdnv55vFkshXgOfxwLAOsetsQAofBQLAOsoFgDWUSwArKNYAFhHsQCwjmIBYB3FAsA6igWAdRQLAOsoFgDW/ReokV962+SllQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "68.0 400 [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 2 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
